{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install -e .. datasets sympy numpy matplotlib seaborn -q  # Install dev version of smolagents + some packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Benchmark date\n",
    "# - set a concrete date:\n",
    "DATE = \"2024-12-26\"\n",
    "# - or use default: today\n",
    "# DATE = None\n",
    "\n",
    "# Evaluation dataset\n",
    "# - the dataset is gated, so you must first visit its page to request access: https://huggingface.co/datasets/smolagents/benchmark-v1\n",
    "EVAL_DATASET = \"GeekAgents/GAIA-web\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Constants and utilities/tools"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import string\n",
    "import warnings\n",
    "from typing import List\n",
    "\n",
    "\n",
    "def normalize_number_str(number_str: str) -> float:\n",
    "    # we replace these common units and commas to allow\n",
    "    # conversion to float\n",
    "    for char in [\"$\", \"%\", \",\"]:\n",
    "        number_str = number_str.replace(char, \"\")\n",
    "    try:\n",
    "        return float(number_str)\n",
    "    except ValueError:\n",
    "        return float(\"inf\")\n",
    "\n",
    "\n",
    "def split_string(\n",
    "    s: str,\n",
    "    char_list: list[str] = [\",\", \";\"],\n",
    ") -> list[str]:\n",
    "    pattern = f\"[{''.join(char_list)}]\"\n",
    "    return re.split(pattern, s)\n",
    "\n",
    "\n",
    "def is_float(element: any) -> bool:\n",
    "    try:\n",
    "        float(element)\n",
    "        return True\n",
    "    except ValueError:\n",
    "        return False\n",
    "\n",
    "\n",
    "def normalize_str(input_str, remove_punct=True) -> str:\n",
    "    \"\"\"\n",
    "    Normalize a string by:\n",
    "    - Removing all white spaces\n",
    "    - Optionally removing punctuation (if remove_punct is True)\n",
    "    - Converting to lowercase\n",
    "    Parameters:\n",
    "    - input_str: str, the string to normalize\n",
    "    - remove_punct: bool, whether to remove punctuation (default: True)\n",
    "    Returns:\n",
    "    - str, the normalized string\n",
    "    \"\"\"\n",
    "    # Remove all white spaces. Required e.g for seagull vs. sea gull\n",
    "    no_spaces = re.sub(r\"\\s\", \"\", input_str)\n",
    "\n",
    "    # Remove punctuation, if specified.\n",
    "    if remove_punct:\n",
    "        translator = str.maketrans(\"\", \"\", string.punctuation)\n",
    "        return no_spaces.lower().translate(translator)\n",
    "    else:\n",
    "        return no_spaces.lower()\n",
    "\n",
    "\n",
    "def extract_numbers(text: str) -> List[str]:\n",
    "    \"\"\"This pattern matches:\n",
    "    - Optional negative sign\n",
    "    - Numbers with optional comma thousand separators\n",
    "    - Optional decimal points with decimal numbers\n",
    "    \"\"\"\n",
    "    pattern = r\"-?(?:\\d{1,3}(?:,\\d{3})+|\\d+)(?:\\.\\d+)?\"\n",
    "\n",
    "    return [el.replace(\",\", \"\") for el in re.findall(pattern, text)]\n",
    "\n",
    "\n",
    "def get_question_score_gaia(\n",
    "    model_answer: str,\n",
    "    ground_truth: str,\n",
    ") -> bool:\n",
    "    \"\"\"Scoring function used to score functions from the GAIA benchmark\"\"\"\n",
    "    if is_float(ground_truth):\n",
    "        normalized_answer = normalize_number_str(str(model_answer))\n",
    "        return normalized_answer == float(ground_truth)\n",
    "\n",
    "    elif any(char in ground_truth for char in [\",\", \";\"]):  # if gt is a list\n",
    "        # question with the fish: normalization removes punct\n",
    "        gt_elems = split_string(ground_truth)\n",
    "        ma_elems = split_string(model_answer)\n",
    "\n",
    "        if len(gt_elems) != len(ma_elems):  # check length is the same\n",
    "            warnings.warn(\"Answer lists have different lengths, returning False.\", UserWarning)\n",
    "            return False\n",
    "\n",
    "        comparisons = []\n",
    "        for ma_elem, gt_elem in zip(ma_elems, gt_elems):  # compare each element as float or str\n",
    "            if is_float(gt_elem):\n",
    "                normalized_ma_elem = normalize_number_str(ma_elem)\n",
    "                comparisons.append(normalized_ma_elem == float(gt_elem))\n",
    "            else:\n",
    "                # we do not remove punct since comparisons can include punct\n",
    "                comparisons.append(\n",
    "                    normalize_str(ma_elem, remove_punct=False) == normalize_str(gt_elem, remove_punct=False)\n",
    "                )\n",
    "        return all(comparisons)\n",
    "\n",
    "    else:  # if gt is a str\n",
    "        return normalize_str(model_answer) == normalize_str(ground_truth)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Score answers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model_id</th>\n",
       "      <th>question</th>\n",
       "      <th>original_question</th>\n",
       "      <th>answer</th>\n",
       "      <th>true_answer</th>\n",
       "      <th>intermediate_steps</th>\n",
       "      <th>start_time</th>\n",
       "      <th>end_time</th>\n",
       "      <th>token_counts</th>\n",
       "      <th>level</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Qwen/Qwen2.5-VL-72B-Instruct</td>\n",
       "      <td>If Eliud Kipchoge could maintain his record-ma...</td>\n",
       "      <td>If Eliud Kipchoge could maintain his record-ma...</td>\n",
       "      <td>17000</td>\n",
       "      <td>17</td>\n",
       "      <td>[TaskStep(task='If Eliud Kipchoge could mainta...</td>\n",
       "      <td>2025-04-02 16:17:38.111258984</td>\n",
       "      <td>2025-04-02 16:18:43.742221117</td>\n",
       "      <td>{'input': 22296, 'output': 595}</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Qwen/Qwen2.5-VL-72B-Instruct</td>\n",
       "      <td>How many studio albums were published by Merce...</td>\n",
       "      <td>How many studio albums were published by Merce...</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>[TaskStep(task='How many studio albums were pu...</td>\n",
       "      <td>2025-04-02 16:17:38.749067068</td>\n",
       "      <td>2025-04-02 16:19:04.685535908</td>\n",
       "      <td>{'input': 29980, 'output': 848}</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Qwen/Qwen2.5-VL-72B-Instruct</td>\n",
       "      <td>What was the volume in m^3 of the fish bag tha...</td>\n",
       "      <td>What was the volume in m^3 of the fish bag tha...</td>\n",
       "      <td>0.177</td>\n",
       "      <td>0.1777</td>\n",
       "      <td>[TaskStep(task='What was the volume in m^3 of ...</td>\n",
       "      <td>2025-04-02 16:17:38.574250937</td>\n",
       "      <td>2025-04-02 16:20:28.700230837</td>\n",
       "      <td>{'input': 59301, 'output': 1616}</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Qwen/Qwen2.5-VL-72B-Instruct</td>\n",
       "      <td>In Series 9, Episode 11 of Doctor Who, the Doc...</td>\n",
       "      <td>In Series 9, Episode 11 of Doctor Who, the Doc...</td>\n",
       "      <td>EXT. THE CASTLE - DAY</td>\n",
       "      <td>THE CASTLE</td>\n",
       "      <td>[TaskStep(task='In Series 9, Episode 11 of Doc...</td>\n",
       "      <td>2025-04-02 16:19:10.391962767</td>\n",
       "      <td>2025-04-02 16:20:41.083353043</td>\n",
       "      <td>{'input': 30810, 'output': 753}</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Qwen/Qwen2.5-VL-72B-Instruct</td>\n",
       "      <td>Of the authors (First M. Last) that worked on ...</td>\n",
       "      <td>Of the authors (First M. Last) that worked on ...</td>\n",
       "      <td>A usability evaluation of Web user interface s...</td>\n",
       "      <td>Mapping Human Oriented Information to Software...</td>\n",
       "      <td>[TaskStep(task='Of the authors (First M. Last)...</td>\n",
       "      <td>2025-04-02 16:18:49.799759150</td>\n",
       "      <td>2025-04-02 16:21:25.836360931</td>\n",
       "      <td>{'input': 57094, 'output': 1303}</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       model_id  \\\n",
       "0  Qwen/Qwen2.5-VL-72B-Instruct   \n",
       "1  Qwen/Qwen2.5-VL-72B-Instruct   \n",
       "2  Qwen/Qwen2.5-VL-72B-Instruct   \n",
       "3  Qwen/Qwen2.5-VL-72B-Instruct   \n",
       "4  Qwen/Qwen2.5-VL-72B-Instruct   \n",
       "\n",
       "                                            question  \\\n",
       "0  If Eliud Kipchoge could maintain his record-ma...   \n",
       "1  How many studio albums were published by Merce...   \n",
       "2  What was the volume in m^3 of the fish bag tha...   \n",
       "3  In Series 9, Episode 11 of Doctor Who, the Doc...   \n",
       "4  Of the authors (First M. Last) that worked on ...   \n",
       "\n",
       "                                   original_question  \\\n",
       "0  If Eliud Kipchoge could maintain his record-ma...   \n",
       "1  How many studio albums were published by Merce...   \n",
       "2  What was the volume in m^3 of the fish bag tha...   \n",
       "3  In Series 9, Episode 11 of Doctor Who, the Doc...   \n",
       "4  Of the authors (First M. Last) that worked on ...   \n",
       "\n",
       "                                              answer  \\\n",
       "0                                              17000   \n",
       "1                                                  3   \n",
       "2                                              0.177   \n",
       "3                              EXT. THE CASTLE - DAY   \n",
       "4  A usability evaluation of Web user interface s...   \n",
       "\n",
       "                                         true_answer  \\\n",
       "0                                                 17   \n",
       "1                                                  3   \n",
       "2                                             0.1777   \n",
       "3                                         THE CASTLE   \n",
       "4  Mapping Human Oriented Information to Software...   \n",
       "\n",
       "                                  intermediate_steps  \\\n",
       "0  [TaskStep(task='If Eliud Kipchoge could mainta...   \n",
       "1  [TaskStep(task='How many studio albums were pu...   \n",
       "2  [TaskStep(task='What was the volume in m^3 of ...   \n",
       "3  [TaskStep(task='In Series 9, Episode 11 of Doc...   \n",
       "4  [TaskStep(task='Of the authors (First M. Last)...   \n",
       "\n",
       "                     start_time                      end_time  \\\n",
       "0 2025-04-02 16:17:38.111258984 2025-04-02 16:18:43.742221117   \n",
       "1 2025-04-02 16:17:38.749067068 2025-04-02 16:19:04.685535908   \n",
       "2 2025-04-02 16:17:38.574250937 2025-04-02 16:20:28.700230837   \n",
       "3 2025-04-02 16:19:10.391962767 2025-04-02 16:20:41.083353043   \n",
       "4 2025-04-02 16:18:49.799759150 2025-04-02 16:21:25.836360931   \n",
       "\n",
       "                       token_counts  level  \n",
       "0   {'input': 22296, 'output': 595}      1  \n",
       "1   {'input': 29980, 'output': 848}      1  \n",
       "2  {'input': 59301, 'output': 1616}      1  \n",
       "3   {'input': 30810, 'output': 753}      1  \n",
       "4  {'input': 57094, 'output': 1303}      1  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import glob\n",
    "\n",
    "import pandas as pd\n",
    "\n",
    "\n",
    "result_dataframes = []\n",
    "for file in glob.glob(\"../../output/*.jsonl\"):\n",
    "    result_dataframes.append(pd.read_json(file, lines=True))\n",
    "result_df = pd.concat(result_dataframes, ignore_index=True)\n",
    "result_df = result_df.loc[result_df[\"level\"] == 1]\n",
    "result_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "result_df[\"is_correct\"] = result_df.apply(lambda x: get_question_score_gaia(x[\"answer\"], x[\"true_answer\"]), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "model_id                      is_correct\n",
       "Qwen/Qwen2.5-VL-72B-Instruct  False         15\n",
       "                              True           3\n",
       "gpt-4o                        False         18\n",
       "                              True           8\n",
       "Name: count, dtype: int64"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>is_correct</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>model_id</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Qwen/Qwen2.5-VL-72B-Instruct</th>\n",
       "      <td>0.166667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>gpt-4o</th>\n",
       "      <td>0.307692</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                              is_correct\n",
       "model_id                                \n",
       "Qwen/Qwen2.5-VL-72B-Instruct    0.166667\n",
       "gpt-4o                          0.307692"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "display(result_df.groupby(\"model_id\")[[\"is_correct\"]].value_counts())\n",
    "\n",
    "result_df.groupby(\"model_id\")[[\"is_correct\"]].mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "agents",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
